package cn.edu.nju.ws.sview.crawler;

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.PriorityQueue;
import java.util.Properties;

import cn.edu.nju.ws.sview.rdf.ResourceFactory;
import cn.edu.nju.ws.sview.rdf.URIResource;
/**
 * 爬虫入口
 * @author Gong Saisai
 */
public class Crawler {
     public final static int THREAD_NUM;
     public final static String USER_AGENT_REQUEST;
     public final static int MAX_REDIRECTION_NUM;
     public final static int CONNECT_TIMEOUT;
     public final static int READ_TIMEOUT;
     public final static long RDF_DOC_EXPIRE_TIME;
     static{
    	 Properties properties = new Properties();
    	 try {
			properties.load(Crawler.class.getClassLoader().getResourceAsStream("conf.properties")) ;
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		THREAD_NUM = Integer.parseInt(properties.getProperty("crawler_thread_num"));
		USER_AGENT_REQUEST  = properties.getProperty("crawler_user_agent_request");
		MAX_REDIRECTION_NUM = Integer.parseInt(properties.getProperty("crawler_max_redirection_num"));
		CONNECT_TIMEOUT = Integer.parseInt(properties.getProperty("crawler_connection_time_out"));
		READ_TIMEOUT = Integer.parseInt(properties.getProperty("crawler_read_time_out"));
		int days = Integer.parseInt(properties.getProperty("rdf_document_expire"));
		RDF_DOC_EXPIRE_TIME = days*24*60*60*1000;
		properties.clear();
     }
     
     private Flag<Boolean> STOP = null;
     private boolean beinStopping ;
     
     /**
      * 完成爬取和解析
      */
     public final static int STATUS_DONE = 1;
     
     /**
      * 正在爬取和解析
      */
     public final static int STATUS_DOING = 2;
     
     /**
      * 等待爬取
      */
     public final static int STATUS_WAITING =3;
     
     /**
      * 没有提交过
      */
     public final static int STATUS_UNSUBMITTED =4;
     
     private static Crawler INSTANCE = null;
     private Object lockOfURIStatus = null;
 	 private HashSet<URIResource> done = null;
 	 private HashSet<URIResource> busy = null;
 	 private PriorityQueue<Element> waiting = null;
 	 private HashMap<URIResource,Element> resource2Elements = null;
 	 
 	 private Crawler(){
 		 //开始结束
 		beinStopping = false;
 		this.STOP = new Flag<Boolean>(false);
 		this.lockOfURIStatus = new Object();
 		this.done = new HashSet<URIResource>();
 		this.busy = new HashSet<URIResource>();
 		this.waiting = new PriorityQueue<Element>();
 		this.resource2Elements =new HashMap<URIResource,Element>();
 		CrawlerMeta meta = new CrawlerMeta(USER_AGENT_REQUEST,MAX_REDIRECTION_NUM,CONNECT_TIMEOUT,READ_TIMEOUT);
 		Thread scheduler = new Thread(new Scheduler(this.done, this.busy,this.waiting,
 		 	    this.resource2Elements,this.lockOfURIStatus,THREAD_NUM,this.STOP,meta));
 		scheduler.start();
 	 }
    
 	 public static synchronized Crawler getInstance(){
 		 if(INSTANCE==null){
 			INSTANCE = new Crawler();
 		 }
 		 return INSTANCE;
 	 }
 	 
 	 /**
 	  * 将URI加入到等待队列，首先会去掉URI中的#
 	  * @param resource
 	  */
 	 public void submitURI(URIResource resource){
 		 if(this.beinStopping){
 			 return ;
 		 }
 		 if(resource.isInternalURI())
 			 return;
 		 String uri = Fetch.removeFragment(resource.getURI());
 		 if(!Fetch.isAcceptableURI(uri))
 			 return;
 		synchronized (this.lockOfURIStatus){
 			URIResource uriResource = ResourceFactory.getInstance().createURIResource(uri) ;
 			if(this.resource2Elements.containsKey(uriResource)){
 				this.promoteWeight(uriResource, Element.LEVEL_1) ;
 				return;
 			}else if(this.done.contains(uriResource)){
 				this.done.remove(uriResource) ;
 			}else if(this.busy.contains(uriResource)){
 			    return;
 			}
 			Element element = new Element(uriResource,Element.NORMAL_WEIGHT);
 			this.resource2Elements.put(uriResource, element);
 			this.waiting.add(element);
 		}
 		 
 	 }
 	 
 	 public void cancel(URIResource resource){
 		 if(this.beinStopping){
 			 return ;
 		 }
 		 if(resource.isInternalURI())
 			 return;
 		 String uri = Fetch.removeFragment(resource.getURI());
 		 if(!Fetch.isAcceptableURI(uri))
 			 return;
 		 synchronized (this.lockOfURIStatus){
 			if(this.resource2Elements.keySet().contains(resource)){
 				Element element  =this.resource2Elements.get(resource);
				this.resource2Elements.remove(resource) ;
				this.waiting.remove(element);
 			}
 		 } 		 
 	 }
 	 
 	 /**
 	  * 通过加入结束符NLL的方式使爬虫退出
 	  */
 	 public void exit(){
 		beinStopping = true;
 		 synchronized(this.lockOfURIStatus){
 			 this.waiting.add(new Element(null,Element.LOW_WEIGHT)) ;
 			 this.resource2Elements.put(null, null) ;
 		 }
 		 while(true){
 			 if(this.STOP.getFlag())
 				 break;
 			 try {
				Thread.sleep(10) ;
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
 		 }
 		 this.busy.clear();
 		 this.done.clear();
 		 this.waiting.clear();
 		 this.resource2Elements.clear();
 	 }
 	 
 	 /**
 	  * 获取URI的解析状态: STATUS_DONE,STATUS_DOING,STATUS_WAITING,STATUS_UNSUBMITTED
 	  * 首先回去掉URI中的hash
 	  * @param resource
 	  * @return URI的解析状态
 	  */
 	 public int checkStatus(URIResource resource){
 		 String uri = Fetch.removeFragment(resource.getURI());
 		 if(!Fetch.isAcceptableURI(uri)){
 			 return STATUS_DONE;
 		 }
 		 if(resource.isInternalURI())
 			 return STATUS_DONE;
 		URIResource uriResource = ResourceFactory.getInstance().createURIResource(uri) ;
 		
 		synchronized (this.lockOfURIStatus){
 		     if(this.done.contains(uriResource))
 		    	 return STATUS_DONE;
 		     else if(this.busy.contains(uriResource))
 		    	 return STATUS_DOING;
 		     else if(this.resource2Elements.keySet().contains(uriResource))
 		    	 return STATUS_WAITING;
 		     else
 		    	 return STATUS_UNSUBMITTED;
 		}
 	 }
 	 
 	 /**
 	  * 提升URI在等待队列中的爬取权重，优先级从到低的顺序为：
 	  * 1) 手动提升某个URI到队列头
 	  * 2) 查询执行
 	  * 3）用户浏览结果以后刷新结果
 	  * 4）广度遍历爬取uri时，多次遇到
 	  * 5) 开发者开发完查询后爬取
 	  * @param resource
 	  */
 	 public void promoteWeight(URIResource resource,int weight){
 		synchronized (this.lockOfURIStatus){
 			if(this.resource2Elements.keySet().contains(resource)){
 				Element old = this.resource2Elements.get(resource) ;
 				Element element = new Element(resource,weight) ;
 				this.resource2Elements.put(resource, element);
 				this.waiting.remove(old);
 				this.waiting.add(element) ;
 			}
 		}
 	 }
 	 public static void main(String[] args) {
		String uri = "http://events.linkeddata.org/ldow2010/";
		URIResource resource = ResourceFactory.getInstance().createURIResource(uri) ;
		Crawler.getInstance().submitURI(resource) ;
		while(Crawler.getInstance().checkStatus(resource)!=Crawler.STATUS_DONE){
			try {
				Thread.sleep(1000) ;
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		Crawler.getInstance().exit();
	}
}
